{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 神经网络部分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "Input_Number=6\n",
    "Out_Number=6\n",
    "X_Data=np.linspace(-1,1,600).reshape(-1,Input_Number)  #Reshape（-1，6）Transform into 100*6 matrice\n",
    "X_Noise=np.random.normal(0,0.05,X_Data.shape)\n",
    "Y_Data=np.sin(X_Data)+X_Noise\n",
    "#-----------------------------------------\n",
    "#Construct a neural network\n",
    "xs=tf.placeholder(tf.float32,[None,Input_Number])\n",
    "ys=tf.placeholder(tf.float32,[None,Out_Number])\n",
    "\n",
    "#Layer1\n",
    "Layer1_Number=3\n",
    "w1=tf.Variable(tf.random_normal([Input_Number,Layer1_Number]))\n",
    "b1=tf.Variable(tf.zeros([1,Layer1_Number])+0.1)\n",
    "l1=tf.nn.relu(tf.matmul(xs,w1)+b1) #b1 use the propagation mechanism\n",
    "\n",
    "#Layer2\n",
    "Layer2_Number=Out_Number\n",
    "w2=tf.Variable(tf.random_normal([Layer1_Number,Layer2_Number]))\n",
    "b2=tf.Variable(tf.zeros([1,Layer2_Number]))\n",
    "l2=tf.nn.tan(tf.matmul(l1,w2)+b2)\n",
    "\n",
    "Loss=tf.reduce_mean(tf.reduce_sum(tf.square(ys-l2),reduction_indices=[1]))\n",
    "H=tf.shape(l2)\n",
    "Train=tf.train.GradientDescentOptimizer(0.0001).minimize(Loss)\n",
    "#Train = tf.train.AdamOptimizer(1e-1).minimize(Loss)\n",
    "\n",
    "Initial=tf.initialize_all_variables()\n",
    "Sess=tf.Session()\n",
    "Sess.run(Initial)\n",
    "\n",
    "for i in range(1000000):\n",
    "    Sess.run(Train,feed_dict={xs:X_Data,ys:Y_Data})\n",
    "    if i % 1000 == 0:\n",
    "        print(Sess.run(Loss, feed_dict={xs: X_Data, ys: Y_Data}))\n",
    "        print(Sess.run(H, feed_dict={xs: X_Data, ys: Y_Data}))\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\util\\tf_should_use.py:170: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "2\n",
      "0\n",
      "0\n",
      "0\n",
      "1\n",
      "3\n",
      "0\n",
      "2\n",
      "0\n",
      "1\n",
      "3\n",
      "1\n",
      "2\n",
      "1\n",
      "2\n",
      "4\n",
      "4\n",
      "3\n",
      "1\n"
     ]
    },
    {
     "ename": "InvalidArgumentError",
     "evalue": "Shape [-1] has negative dimensions\n\t [[Node: Placeholder_41 = Placeholder[dtype=DT_FLOAT, shape=[?], _device=\"/job:localhost/replica:0/task:0/cpu:0\"]()]]\n\nCaused by op 'Placeholder_41', defined at:\n  File \"D:\\Miniconda\\Miniconda1\\lib\\runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\traitlets\\config\\application.py\", line 658, in launch_instance\n    app.start()\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 477, in start\n    ioloop.IOLoop.instance().start()\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\zmq\\eventloop\\ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tornado\\ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tornado\\stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tornado\\stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 235, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 533, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2698, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2802, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2862, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-17-809ea1a88e2b>\", line 341, in <module>\n    q_network = DeepQNetwork()\n  File \"<ipython-input-17-809ea1a88e2b>\", line 93, in __init__\n    self.create_network()\n  File \"<ipython-input-17-809ea1a88e2b>\", line 111, in create_network\n    self.q_target = tf.placeholder(shape=[None], dtype=tf.float32)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\ops\\array_ops.py\", line 1530, in placeholder\n    return gen_array_ops._placeholder(dtype=dtype, shape=shape, name=name)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\ops\\gen_array_ops.py\", line 1954, in _placeholder\n    name=name)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\", line 767, in apply_op\n    op_def=op_def)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 2506, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 1269, in __init__\n    self._traceback = _extract_stack()\n\nInvalidArgumentError (see above for traceback): Shape [-1] has negative dimensions\n\t [[Node: Placeholder_41 = Placeholder[dtype=DT_FLOAT, shape=[?], _device=\"/job:localhost/replica:0/task:0/cpu:0\"]()]]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[1;32mD:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1138\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1139\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1140\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1120\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1121\u001b[1;33m                                  status, run_metadata)\n\u001b[0m\u001b[0;32m   1122\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Miniconda\\Miniconda1\\lib\\contextlib.py\u001b[0m in \u001b[0;36m__exit__\u001b[1;34m(self, type, value, traceback)\u001b[0m\n\u001b[0;32m     87\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 88\u001b[1;33m                 \u001b[0mnext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     89\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\framework\\errors_impl.py\u001b[0m in \u001b[0;36mraise_exception_on_not_ok_status\u001b[1;34m()\u001b[0m\n\u001b[0;32m    465\u001b[0m           \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpywrap_tensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstatus\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 466\u001b[1;33m           pywrap_tensorflow.TF_GetCode(status))\n\u001b[0m\u001b[0;32m    467\u001b[0m   \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mInvalidArgumentError\u001b[0m: Shape [-1] has negative dimensions\n\t [[Node: Placeholder_41 = Placeholder[dtype=DT_FLOAT, shape=[?], _device=\"/job:localhost/replica:0/task:0/cpu:0\"]()]]",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-17-809ea1a88e2b>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    340\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"__main__\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    341\u001b[0m     \u001b[0mq_network\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mDeepQNetwork\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 342\u001b[1;33m     \u001b[0mq_network\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpay\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-17-809ea1a88e2b>\u001b[0m in \u001b[0;36mpay\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    304\u001b[0m         \u001b[1;33m:\u001b[0m\u001b[1;32mreturn\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    305\u001b[0m         \"\"\"\n\u001b[1;32m--> 306\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    307\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    308\u001b[0m         \u001b[1;31m# 显示 R 矩阵。\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-17-809ea1a88e2b>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    286\u001b[0m             \u001b[1;31m# 先观察一段时间累积足够的记忆在进行训练。\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    287\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep_index\u001b[0m \u001b[1;33m>\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOBSERVE\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 288\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexperience_replay\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    289\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    290\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep_index\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m10000\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-17-809ea1a88e2b>\u001b[0m in \u001b[0;36mexperience_replay\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    251\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    252\u001b[0m                 \u001b[0mq_target\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mq_value\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 253\u001b[1;33m             \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mq_target\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mq_target\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mq_target\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    254\u001b[0m         _, cost, reward = self.session.run([self.train_op, self.loss, self.reward_action],\n\u001b[0;32m    255\u001b[0m                                            feed_dict={self.q_eval_input: batch_state,\n",
      "\u001b[1;32mD:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    787\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    788\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 789\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    790\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    791\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    995\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    996\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m--> 997\u001b[1;33m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[0;32m    998\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    999\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1130\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1131\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[1;32m-> 1132\u001b[1;33m                            target_list, options, run_metadata)\n\u001b[0m\u001b[0;32m   1133\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1134\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[1;32mD:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1150\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1151\u001b[0m           \u001b[1;32mpass\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1152\u001b[1;33m       \u001b[1;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1153\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1154\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mInvalidArgumentError\u001b[0m: Shape [-1] has negative dimensions\n\t [[Node: Placeholder_41 = Placeholder[dtype=DT_FLOAT, shape=[?], _device=\"/job:localhost/replica:0/task:0/cpu:0\"]()]]\n\nCaused by op 'Placeholder_41', defined at:\n  File \"D:\\Miniconda\\Miniconda1\\lib\\runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\traitlets\\config\\application.py\", line 658, in launch_instance\n    app.start()\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 477, in start\n    ioloop.IOLoop.instance().start()\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\zmq\\eventloop\\ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tornado\\ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tornado\\stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tornado\\stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 235, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 533, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2698, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2802, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2862, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-17-809ea1a88e2b>\", line 341, in <module>\n    q_network = DeepQNetwork()\n  File \"<ipython-input-17-809ea1a88e2b>\", line 93, in __init__\n    self.create_network()\n  File \"<ipython-input-17-809ea1a88e2b>\", line 111, in create_network\n    self.q_target = tf.placeholder(shape=[None], dtype=tf.float32)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\ops\\array_ops.py\", line 1530, in placeholder\n    return gen_array_ops._placeholder(dtype=dtype, shape=shape, name=name)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\ops\\gen_array_ops.py\", line 1954, in _placeholder\n    name=name)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\", line 767, in apply_op\n    op_def=op_def)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 2506, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"D:\\Miniconda\\Miniconda1\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 1269, in __init__\n    self._traceback = _extract_stack()\n\nInvalidArgumentError (see above for traceback): Shape [-1] has negative dimensions\n\t [[Node: Placeholder_41 = Placeholder[dtype=DT_FLOAT, shape=[?], _device=\"/job:localhost/replica:0/task:0/cpu:0\"]()]]\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import random\n",
    "\n",
    "\n",
    "class DeepQNetwork:\n",
    "    r = np.array([[-1, -1, -1, -1, 0, -1],\n",
    "                  [-1, -1, -1, 0, -1, 100.0],\n",
    "                  [-1, -1, -1, 0, -1, -1],\n",
    "                  [-1, 0, 0, -1, 0, -1],\n",
    "                  [0, -1, -1, 1, -1, 100],\n",
    "                  [-1, 0, -1, -1, 0, 100],\n",
    "                  ])\n",
    "\n",
    "    # 执行步数。\n",
    "    step_index = 0\n",
    "\n",
    "    # 状态数。\n",
    "    state_num = 6\n",
    "\n",
    "    # 动作数。\n",
    "    action_num = 6\n",
    "\n",
    "    # 训练之前观察多少步。\n",
    "    OBSERVE = 1000.\n",
    "\n",
    "    # 选取的小批量训练样本数。\n",
    "    BATCH = 20\n",
    "\n",
    "    # epsilon 的最小值，当 epsilon 小于该值时，将不在随机选择行为。\n",
    "    FINAL_EPSILON = 0.0001\n",
    "\n",
    "    # epsilon 的初始值，epsilon 逐渐减小。\n",
    "    INITIAL_EPSILON = 0.1\n",
    "\n",
    "    # epsilon 衰减的总步数。\n",
    "    EXPLORE = 3000000.\n",
    "\n",
    "    # 探索模式计数。\n",
    "    epsilon = 0\n",
    "\n",
    "    # 训练步数统计。\n",
    "    learn_step_counter = 0\n",
    "\n",
    "    # 学习率。\n",
    "    learning_rate = 0.001\n",
    "\n",
    "    # γ经验折损率。\n",
    "    gamma = 0.9\n",
    "\n",
    "    # 记忆上限。\n",
    "    memory_size = 5000\n",
    "\n",
    "    # 当前记忆数。\n",
    "    memory_counter = 0\n",
    "\n",
    "    # 保存观察到的执行过的行动的存储器，即：曾经经历过的记忆。\n",
    "    replay_memory_store = deque()\n",
    "\n",
    "    # 生成一个状态矩阵（6 X 6），每一行代表一个状态。\n",
    "    state_list = None\n",
    "\n",
    "    # 生成一个动作矩阵。\n",
    "    action_list = None\n",
    "\n",
    "    # q_eval 网络。\n",
    "    q_eval_input = None\n",
    "    action_input = None\n",
    "    q_target = None\n",
    "    q_eval = None\n",
    "    predict = None\n",
    "    loss = None\n",
    "    train_op = None\n",
    "    cost_his = None\n",
    "    reward_action = None\n",
    "\n",
    "    # tensorflow 会话。\n",
    "    session = None\n",
    "\n",
    "    def __init__(self, learning_rate=0.001, gamma=0.9, memory_size=5000):\n",
    "        self.learning_rate = learning_rate\n",
    "        self.gamma = gamma\n",
    "        self.memory_size = memory_size\n",
    "\n",
    "        # 初始化成一个 6 X 6 的状态矩阵。\n",
    "        self.state_list = np.identity(self.state_num)\n",
    "\n",
    "        # 初始化成一个 6 X 6 的动作矩阵。\n",
    "        self.action_list = np.identity(self.action_num)\n",
    "\n",
    "        # 创建神经网络。\n",
    "        self.create_network()\n",
    "\n",
    "        # 初始化 tensorflow 会话。\n",
    "        self.session = tf.InteractiveSession()\n",
    "\n",
    "        # 初始化 tensorflow 参数。\n",
    "        self.session.run(tf.initialize_all_variables())\n",
    "\n",
    "        # 记录所有 loss 变化。\n",
    "        self.cost_his = []\n",
    "\n",
    "    def create_network(self):\n",
    "        \"\"\"\n",
    "        创建神经网络。\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        self.q_eval_input = tf.placeholder(shape=[None, self.state_num], dtype=tf.float32)\n",
    "        self.action_input = tf.placeholder(shape=[None, self.action_num], dtype=tf.float32)\n",
    "        self.q_target = tf.placeholder(shape=[None], dtype=tf.float32)\n",
    "\n",
    "        neuro_layer_1 = 3\n",
    "        w1 = tf.Variable(tf.random_normal([self.state_num, neuro_layer_1]))\n",
    "        b1 = tf.Variable(tf.zeros([1, neuro_layer_1]) + 0.1)\n",
    "        l1 = tf.nn.relu(tf.matmul(self.q_eval_input, w1) + b1)\n",
    "\n",
    "        w2 = tf.Variable(tf.random_normal([neuro_layer_1, self.action_num]))\n",
    "        b2 = tf.Variable(tf.zeros([1, self.action_num]) + 0.1)\n",
    "        self.q_eval = tf.matmul(l1, w2) + b2\n",
    "\n",
    "        # 取出当前动作的得分。\n",
    "        self.reward_action = tf.reduce_sum(tf.multiply(self.q_eval, self.action_input), reduction_indices=1)\n",
    "        self.loss = tf.reduce_mean(tf.square((self.q_target - self.reward_action)))\n",
    "        self.train_op = tf.train.GradientDescentOptimizer(self.learning_rate).minimize(self.loss)\n",
    "\n",
    "        self.predict = tf.argmax(self.q_eval, 1)\n",
    "    \n",
    "    def select_action(self, state_index):\n",
    "        \"\"\"\n",
    "        根据策略选择动作。\n",
    "        :param state_index: 当前状态。\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        current_state = self.state_list[state_index:state_index + 1]\n",
    "\n",
    "        if np.random.uniform() < self.epsilon:\n",
    "            current_action_index = np.random.randint(0, self.action_num)\n",
    "        else:\n",
    "            actions_value = self.session.run(self.q_eval, feed_dict={self.q_eval_input: current_state})\n",
    "            action = np.argmax(actions_value)\n",
    "            current_action_index = action\n",
    "\n",
    "        # 开始训练后，在 epsilon 小于一定的值之前，将逐步减小 epsilon。\n",
    "        if self.step_index > self.OBSERVE and self.epsilon > self.FINAL_EPSILON:\n",
    "            self.epsilon -= (self.INITIAL_EPSILON - self.FINAL_EPSILON) / self.EXPLORE\n",
    "\n",
    "        return current_action_index\n",
    "    #***************************************************************\n",
    "    def save_store(self, current_state_index, current_action_index, current_reward, next_state_index, done):\n",
    "        \"\"\"\n",
    "        保存记忆。\n",
    "        :param current_state_index: 当前状态 index。\n",
    "        :param current_action_index: 动作 index。\n",
    "        :param current_reward: 奖励。\n",
    "        :param next_state_index: 下一个状态 index。\n",
    "        :param done: 是否结束。\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        current_state = self.state_list[current_state_index:current_state_index + 1]\n",
    "        current_action = self.action_list[current_action_index:current_action_index + 1]\n",
    "        next_state = self.state_list[next_state_index:next_state_index + 1]\n",
    "        # 记忆动作(当前状态， 当前执行的动作， 当前动作的得分，下一个状态)。\n",
    "        self.replay_memory_store.append((\n",
    "            current_state,\n",
    "            current_action,\n",
    "            current_reward,\n",
    "            next_state,\n",
    "            done))\n",
    "\n",
    "        # 如果超过记忆的容量，则将最久远的记忆移除。\n",
    "        if len(self.replay_memory_store) > self.memory_size:\n",
    "            self.replay_memory_store.popleft()\n",
    "\n",
    "        self.memory_counter += 1\n",
    "    #***************************************************************\n",
    "    def step(self, state, action):\n",
    "        \"\"\"\n",
    "        执行动作。\n",
    "        :param state: 当前状态。\n",
    "        :param action: 执行的动作。\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        reward = self.r[state][action]\n",
    "\n",
    "        next_state = action\n",
    "\n",
    "        done = False\n",
    "\n",
    "        if action == 5:\n",
    "            done = True\n",
    "\n",
    "        return next_state, reward, done\n",
    "    #***************************************************************\n",
    "    def experience_replay(self):\n",
    "        \"\"\"\n",
    "        记忆回放。\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        # 随机选择一小批记忆样本。\n",
    "        batch = self.BATCH if self.memory_counter > self.BATCH else self.memory_counter\n",
    "        minibatch = random.sample(self.replay_memory_store, batch)\n",
    "\n",
    "        batch_state = None\n",
    "        batch_action = None\n",
    "        batch_reward = None\n",
    "        batch_next_state = None\n",
    "        batch_done = None\n",
    "\n",
    "        for index in range(len(minibatch)):\n",
    "            if batch_state is None:\n",
    "                batch_state = minibatch[index][0]\n",
    "            elif batch_state is not None:\n",
    "                batch_state = np.vstack((batch_state, minibatch[index][0]))\n",
    "\n",
    "            if batch_action is None:\n",
    "                batch_action = minibatch[index][1]\n",
    "            elif batch_action is not None:\n",
    "                batch_action = np.vstack((batch_action, minibatch[index][1]))\n",
    "\n",
    "            if batch_reward is None:\n",
    "                batch_reward = minibatch[index][2]\n",
    "            elif batch_reward is not None:\n",
    "                batch_reward = np.vstack((batch_reward, minibatch[index][2]))\n",
    "            if batch_next_state is None:\n",
    "                batch_next_state = minibatch[index][3]\n",
    "            elif batch_next_state is not None:\n",
    "                batch_next_state = np.vstack((batch_next_state, minibatch[index][3]))\n",
    "\n",
    "            if batch_done is None:\n",
    "                batch_done = minibatch[index][4]\n",
    "            elif batch_done is not None:\n",
    "                batch_done = np.vstack((batch_done, minibatch[index][4]))\n",
    "\n",
    "        # q_next：下一个状态的 Q 值。\n",
    "        q_next = self.session.run(self.q_eval, feed_dict={self.q_eval_input: batch_next_state})\n",
    "        q_target = []\n",
    "        for i in range(len(minibatch)):\n",
    "            # 当前即时得分。\n",
    "            current_reward = batch_reward[i][0]\n",
    "\n",
    "            # # 游戏是否结束。\n",
    "            # current_done = batch_done[i][0]\n",
    "\n",
    "            # 更新 Q 值。\n",
    "            q_value = current_reward + self.gamma * np.max(q_next[0][i])\n",
    "\n",
    "            # 当得分小于 0 时，表示走了不可走的位置。\n",
    "            if current_reward < 0:\n",
    "                q_target.append(current_reward)\n",
    "            else:\n",
    "                q_target.append(q_value)\n",
    "            print(self.session.run(self.q_target),feed_dict={self.q_target:q_target})\n",
    "        _, cost, reward = self.session.run([self.train_op, self.loss, self.reward_action],\n",
    "                                           feed_dict={self.q_eval_input: batch_state,\n",
    "                                                      self.action_input: batch_action,\n",
    "                                                      self.q_target: q_target})\n",
    "\n",
    "        self.cost_his.append(cost)\n",
    "\n",
    "        # if self.step_index % 1000 == 0:\n",
    "        #     print(\"loss:\", cost)\n",
    "\n",
    "        self.learn_step_counter += 1\n",
    "\n",
    "    def train(self):\n",
    "        \"\"\"\n",
    "        训练。\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        # 初始化当前状态。\n",
    "        current_state = np.random.randint(0, self.action_num - 1)\n",
    "\n",
    "        self.epsilon = self.INITIAL_EPSILON\n",
    "\n",
    "        while True:\n",
    "            # 选择动作。\n",
    "            action = self.select_action(current_state)\n",
    "\n",
    "            # 执行动作，得到：下一个状态，执行动作的得分，是否结束。\n",
    "            next_state, reward, done = self.step(current_state, action)\n",
    "\n",
    "            # 保存记忆。\n",
    "            self.save_store(current_state, action, reward, next_state, done)\n",
    "\n",
    "            # 先观察一段时间累积足够的记忆在进行训练。\n",
    "            if self.step_index > self.OBSERVE:\n",
    "                self.experience_replay()\n",
    "\n",
    "            if self.step_index > 10000:\n",
    "                break\n",
    "\n",
    "            if done:\n",
    "                current_state = np.random.randint(0, self.action_num - 1)\n",
    "                print(current_state)\n",
    "            else:\n",
    "                current_state = next_state\n",
    "\n",
    "            self.step_index += 1\n",
    "\n",
    "    def pay(self):\n",
    "        \"\"\"\n",
    "        运行并测试。\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        self.train()\n",
    "\n",
    "        # 显示 R 矩阵。\n",
    "        print(self.r)\n",
    "\n",
    "        for index in range(5):\n",
    "\n",
    "            start_room = index\n",
    "\n",
    "            print(\"#############################\", \"Agent 在\", start_room, \"开始行动\", \"#############################\")\n",
    "\n",
    "            current_state = start_room\n",
    "\n",
    "            step = 0\n",
    "\n",
    "            target_state = 5\n",
    "\n",
    "            while current_state != target_state:\n",
    "                out_result = self.session.run(self.q_eval, feed_dict={\n",
    "                    self.q_eval_input: self.state_list[current_state:current_state + 1]})\n",
    "\n",
    "                next_state = np.argmax(out_result[0])\n",
    "\n",
    "                print(\"Agent 由\", current_state, \"号房间移动到了\", next_state, \"号房间\")\n",
    "\n",
    "                current_state = next_state\n",
    "\n",
    "                step += 1\n",
    "\n",
    "            print(\"Agent 在\", start_room, \"号房间开始移动了\", step, \"步到达了目标房间 5\")\n",
    "\n",
    "            print(\"#############################\", \"Agent 在\", 5, \"结束行动\", \"#############################\")\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    q_network = DeepQNetwork()\n",
    "    q_network.pay()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "76"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
